#!/usr/bin/env python
# coding=utf-8
import importlib.resources as resources
import json
import logging
import pathlib
import re
import time as ptime
from datetime import datetime, time
from quant import trader
import pytz
import execjs
import pandas as pd
import requests
from bs4 import BeautifulSoup, element
from tqdm import tqdm
from easydict import EasyDict

HEADERS = {
    'User-Agent':
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 '
        'Safari/537.36 '
}

PROXIES = {
    'http': '',
    'https': '',
}


def auth():
    auth_file = pathlib.Path('~/.cache/quant/.auth.json').expanduser()
    cred = EasyDict(json.load(auth_file.open(mode='r')))
    key = '397151C04723421F'
    ctx = execjs.compile(resources.read_text(__package__, 'jisilu.js'))
    s = requests.Session()
    s.post('https://www.jisilu.cn/account/ajax/login_process/',
           data={
               '_post_type': 'ajax',
               'aes': 1,
               'net_auto_login': '1',
               'password': ctx.call('jslencode', cred.jisilu.password, key),
               'return_url': 'https://www.jisilu.cn/',
               'user_name': ctx.call('jslencode', cred.jisilu.username, key),
           },
           proxies=PROXIES,
           headers=HEADERS)
    return s


def get_conbond_detail(code: str, client: requests.Session,
                       cache_dir: pathlib.Path):
    conbond_json = cache_dir.joinpath('%s.json' % code)
    conbond_html = cache_dir.joinpath('%s.html' % code)
    if conbond_json.exists() and conbond_html.exists():
        logging.info('reading %s from cache' % code)
        json_data = json.load(conbond_json.open(mode='r'))
        html = conbond_html.open(mode='r').read()
        ptime.sleep(1)
    else:
        logging.info('reading %s from jsl' % code)
        url = 'https://www.jisilu.cn/data/cbnew/detail_hist/%s?___jsl=LST___t=%s' % (
            code, int(datetime.now().timestamp() * 1000))
        response = client.post(url, headers=HEADERS)
        json_data = json.loads(response.content.decode('utf-8'))
        json.dump(json_data,
                  conbond_json.open(mode='w'),
                  ensure_ascii=False,
                  indent=4)

        url = 'https://www.jisilu.cn/data/convert_bond_detail/%s' % code
        response = client.post(url, headers=HEADERS)
        html = response.content.decode('utf-8')
        conbond_html.open(mode='w').write(html)

    return json_data, html


def parse_conbond_info(html: str):
    soup = BeautifulSoup(html, 'html.parser')
    for redeem_info in soup.select('#redeem_tc'):
        for table in redeem_info.find_all('table'):
            for row in table.find_all('tr'):
                cells = row.find_all('td')
                logging.info(
                    '%s, %s, %s' %
                    (cells[1].string, cells[2].string, cells[3].string))


def get_all_conbonds(client: requests.Session, cache_dir: pathlib.Path):
    trading_json = cache_dir.joinpath('trading_conbonds.json')
    delisted_json = cache_dir.joinpath('delisted_conbonds.json')
    if trading_json.exists() and delisted_json.exists():
        logging.info('reading all conbonds from cache')
        trading = json.load(trading_json.open(mode='r'))
        delisted = json.load(delisted_json.open(mode='r'))
    else:
        logging.info('reading all conbonds from jsl')
        ts = int(
            datetime.fromordinal(
                datetime.now().date().toordinal()).timestamp() * 1000)
        url = 'https://www.jisilu.cn/data/cbnew/cb_list_new/?___jsl=LST___t=%s' % ts
        response = client.post(url, headers=HEADERS)
        trading = json.loads(response.content.decode('utf-8'))
        json.dump(trading,
                  trading_json.open(mode='w'),
                  ensure_ascii=False,
                  indent=4)

        url = 'https://www.jisilu.cn/data/cbnew/delisted/?___jsl=LST___t=%s' % ts
        response = client.post(url, headers=HEADERS)
        delisted = json.loads(response.content.decode('utf-8'))
        json.dump(delisted,
                  delisted_json.open(mode='w'),
                  ensure_ascii=False,
                  indent=4)
    conbonds = {'order_book_id': []}
    for row in trading['rows']:
        conbonds['order_book_id'].append(row['id'])
    for row in delisted['rows']:
        conbonds['order_book_id'].append(row['id'])
    df = pd.DataFrame(conbonds)
    return df


def refresh_conbond(client: requests.Session, cache_dir: pathlib.Path):
    all_conbonds = get_all_conbonds(client, cache_dir)
    logging.info(len(all_conbonds))
    for code in tqdm(all_conbonds.order_book_id.tolist()):
        logging.info('crawling %s' % code)
        jsondata, html = get_conbond_detail(code, client, cache_dir)
        parse_conbond_info(html)


def refresh_delisted(client: requests.Session, lst: int):
    url = 'https://www.jisilu.cn/data/cbnew/delisted/?___jsl=LST___t=%s' % lst
    response = client.post(url, proxies=PROXIES, headers=HEADERS)
    delisted_json = json.loads(response.content.decode('utf-8'))
    rows = [row['cell'] for row in delisted_json['rows']]
    delisted = pd.DataFrame().from_records(rows)
    delisted.rename(columns={
        'bond_id': 'order_book_id',
        'bond_nm': 'symbol',
        'stock_id': 'stock_code',
        'price': 'open',
        'maturity_dt': 'maturity_date',
        'delist_dt': 'de_listed_date',
        'redeem_dt': 'record_date',
    },
        inplace=True)
    delisted['stop_trading_date'] = delisted.de_listed_date
    delisted.set_index('order_book_id', inplace=True)
    return delisted_json, delisted


def refresh_cb_index(cache_dir):
    logging.info("Refreshing cb_index")
    import os
    all_proxy = os.getenv('ALL_PROXY')
    assert all_proxy is None or all_proxy == '', all_proxy
    from requests_html import HTMLSession
    session = HTMLSession()
    url = 'https://www.jisilu.cn/data/cbnew/cb_index/'
    # Not working somehow, need to unset ALL_PROXY, HTTP_PROXY, HTTPS_PROXY etc. for it to work.
    # or maybe it is affecting the headless chrome?
    response = session.get(url, headers=HEADERS, proxies=PROXIES)
    response.html.render(timeout=30)
    html = response.html.html
    soup = BeautifulSoup(html, 'html.parser')
    table = soup.find('table', attrs={'id': 'table_cb_index'})
    assert table is not None

    data = []
    assert isinstance(table, element.Tag)
    thead = table.find('thead')
    assert thead is not None
    row = thead.find('tr')
    assert row is not None and isinstance(row, element.Tag)
    cols = row.find_all('th')
    assert cols is not None
    headers = [ele.text.strip() for ele in cols]

    tbody = soup.find('tbody', attrs={'id': 'table_cb_index_body'})
    assert tbody is not None
    assert isinstance(tbody, element.Tag)
    rows = tbody.find_all('tr')
    for row in rows:
        cols = row.find_all('td')
        cols = [ele.text.strip() for ele in cols]
        data.append(cols)

    df = pd.DataFrame.from_records(data, columns=headers)
    df['order_book_id'] = 'CB.IDX'
    df['datetime'] = pd.to_datetime(df.日期).dt.tz_localize(
        pytz.timezone('Asia/Hong_Kong'))
    df['volume'] = 10000000
    for f in ['open', 'close', 'low', 'high']:
        df[f] = df.指数
    df = df[[
        'order_book_id',
        'datetime',
        'open',
        'close',
        'low',
        'high',
        'volume',
    ]]
    df.set_index(['datetime', 'order_book_id'], inplace=True)
    logging.info('Updating cb_index')
    with cache_dir.joinpath('cb_index.html').open(
            mode='w', encoding='UTF-8') as f:
        f.write(html)
    df.to_csv(cache_dir.joinpath('cb_index.csv'))


def refresh_cb_list_new(jsl, lst, dt, cache_dir):
    if lst is None:
        logging.info('Read jisilu json for %s' % dt)
        jisilu_data = json.load(cache_dir.joinpath('%s.json' % dt).open(mode='r'))
    else:
        logging.info('Read cb_list_new from jisilu for %s' % dt)
        url = 'https://www.jisilu.cn/data/cbnew/cb_list_new/?___jsl=LST___t=%s' % lst
        payload = {'listed': 'Y'}
        response = jsl.post(url, data=payload, proxies=PROXIES, headers=HEADERS)
        jisilu_data = json.loads(response.content.decode('utf-8'))
        with open(cache_dir.joinpath('%s.json' % dt),
                  'w',
                  encoding='utf-8') as f:
            json.dump(jisilu_data, f, ensure_ascii=False, indent=4)
    jd = {}
    for row in jisilu_data['rows']:
        jd[row['id']] = row['cell']
    df = pd.DataFrame.from_dict(jd, orient='index')
    df = df[df.qflag2 != 'Q']  # 过滤仅机构可买
    df['bond_type'] = df.btype.apply(lambda bt: 'cb' if bt == 'C' else ('eb' if bt == 'E' else 'na'))
    df['stop_trading_date'] = df.bond_nm_tip.apply(
        lambda fr: '0000-00-00' if pd.isna(fr) or '最后交易日：' not in fr else
        '{:04d}-{:02d}-{:02d}'.format(*(int(n) for n in list(
            re.match(r'最后交易日：(\d+)年(\d+)月(\d+)日', fr).groups()))))
    df['de_listed_date'] = df.stop_trading_date
    df['record_date'] = df.bond_nm_tip.apply(
        lambda fr: '0000-00-00' if pd.isna(fr) or '最后转股日：' not in fr else
        '{:04d}-{:02d}-{:02d}'.format(*(int(n) for n in list(
            re.match(r'.*最后转股日：(\d+)年(\d+)月(\d+)日', fr, re.DOTALL).groups()))))
    df['price'] = df.price.astype(float)
    return df


def refresh_redeem(jsl, lst, dt, cache_dir):
    if lst is None:
        logging.info('Read redeem json for %s' % dt)
        redeem_data = json.load(cache_dir.joinpath('redeem-%s.json' % dt).open(mode='r'))
    else:
        logging.info('Read redeem from jisilu for %s' % dt)
        url = 'https://www.jisilu.cn/data/cbnew/redeem_list/?___jsl=LST___t=%s' % lst
        response = jsl.post(url, proxies=PROXIES, headers=HEADERS)
        redeem_data = json.loads(response.content.decode('utf-8'))
        with open(cache_dir.joinpath('redeem-%s.json' % dt),
                  'w',
                  encoding='utf-8') as f:
            json.dump(redeem_data, f, ensure_ascii=False, indent=4)
    df_redeem = pd.DataFrame.from_records(
        [row['cell'] for row in redeem_data['rows']])
    return df_redeem


def refresh_put(jsl, lst, dt, cache_dir):
    if lst is None:
        logging.info('Read put json for %s' % dt)
        put_data = json.load(cache_dir.joinpath('put-%s.json' % dt).open(mode='r'))
    else:
        logging.info('Read huishou from jisilu for %s' % dt)
        url = 'https://www.jisilu.cn/data/cbnew/huishou_list/?___jsl=LST___t=%s' % lst
        response = jsl.post(url, proxies=PROXIES, headers=HEADERS)
        put_data = json.loads(response.content.decode('utf-8'))
        with open(cache_dir.joinpath('put-%s.json' % dt),
                  'w',
                  encoding='utf-8') as f:
            json.dump(put_data, f, ensure_ascii=False, indent=4)
    df_put = pd.DataFrame.from_records(
        [row['cell'] for row in put_data['rows']])
    return df_put


def refresh_now(cache_dir: pathlib.Path, dt=None):
    assert cache_dir is not None
    now = datetime.now()
    if dt is None:
        lst = int(now.timestamp() * 1000)
        dt = trader.Trader.trading_date_in_n(str(now.date()), -1)
        if now.time() > time(9, 30, 0) and trader.Trader.trading_date_in_n(
                dt, 1) == str(now.date()):
            dt = now.date()
    else:
        lst = None
    fcsv = cache_dir.joinpath('%s.csv' % dt)
    if fcsv.exists():
        logging.info('Read %s from cache' % dt)
        df = pd.read_csv(fcsv,
                         index_col=['datetime', 'order_book_id'],
                         parse_dates=['datetime'])
        logging.info('Read delisted from cache')
        delisted = pd.read_csv(cache_dir.joinpath('delisted.csv'),
                               index_col=['order_book_id'])
    else:
        jsl = auth()
        df = refresh_cb_list_new(jsl, lst, dt, cache_dir)
        df_redeem = refresh_redeem(jsl, lst, dt, cache_dir)
        df = df.merge(df_redeem[['bond_id', 'redeem_real_days']],
                      on=['bond_id'])
        df_put = refresh_put(jsl, lst, dt, cache_dir)
        df = df.merge(df_put[['bond_id', 'time']], on=['bond_id'])

        # Why ytm is '-' sometimes?
        df.rename(columns={
            'bond_id': 'order_book_id',
            'bond_nm': 'symbol',
            'stock_id': 'stock_code',
            'price': 'open',
            'sprice': 'stock_price',
            'premium_rt': 'convert_premium_rate',
            'short_maturity_dt': 'maturity_date',
            'dblow': 'double_low_factor',
            'convert_value': 'conversion_value',
            'ytm_rt': 'yield_to_maturity',
            'redeem_real_days': 'call_qualified_days',
            'time': 'put_qualified_days',
            'curr_iss_amt': 'remaining_size',
        },
            inplace=True)
        df['maturity_date'] = '20' + df.maturity_date
        df['convert_premium_rate'] = df.convert_premium_rate / 100
        df['datetime'] = dt
        df['datetime'] = pd.to_datetime(df.datetime).dt.tz_localize(
            pytz.timezone('Asia/Hong_Kong'))
        df.set_index(['datetime', 'order_book_id'], inplace=True)

        if lst is None:
            # Reprocessing from json for specific date
            logging.info('Updating %s' % fcsv)
            df.to_csv(fcsv)
            delisted = pd.read_csv(cache_dir.joinpath('delisted.csv'))
        else:
            delisted_json, delisted = refresh_delisted(jsl, lst)
            if (now.time() < time(9, 30, 0) or now.time() > time(
                    15, 0, 0)):
                refresh_cb_index(cache_dir)
                logging.info('Updating jsl files for %s...' % dt)
                with cache_dir.joinpath('delisted.json').open(mode='w') as f:
                    json.dump(delisted_json, f, ensure_ascii=False, indent=4)
                delisted.to_csv(cache_dir.joinpath('delisted.csv'))
                df.to_csv(fcsv)

    ins_cols = [
        'maturity_date', 'record_date', 'stop_trading_date', 'de_listed_date'
    ]
    return pd.concat([
        df.reset_index().drop(
            columns=['datetime']).set_index('order_book_id')[ins_cols],
        delisted[ins_cols]
    ]), df
